<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>可插拔的相似度算法 | Elasticsearch: 权威指南 | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: 权威指南">
<link rel="up" href="controlling-relevance.html" title="控制相关度">
<link rel="prev" href="script-score.html" title="脚本评分">
<link rel="next" href="changing-similarities.html" title="更改相似度">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Definitive Guide">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="zh_cn">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<b>请注意:</b><br>本书基于 Elasticsearch 2.x 版本，有些内容可能已经过时。
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: 权威指南</a></span>
»
<span class="breadcrumb-link"><a href="search-in-depth.html">深入搜索</a></span>
»
<span class="breadcrumb-link"><a href="controlling-relevance.html">控制相关度</a></span>
»
<span class="breadcrumb-node">可插拔的相似度算法</span>
</div>
<div class="navheader">
<span class="prev">
<a href="script-score.html">« 脚本评分</a>
</span>
<span class="next">
<a href="changing-similarities.html">更改相似度 »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="pluggable-similarites"></a>可插拔的相似度算法<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elasticsearch-cn/elasticsearch-definitive-guide/edit/cn/170_Relevance/70_Pluggable_similarities.asciidoc">edit</a>
</h2>
</div></div></div>
<p>在进一步讨论相关度和评分之前，我们会以一个更高级的话题结束本章节的内容：可插拔的相似度算法（Pluggable Similarity Algorithms）。 Elasticsearch 将 <a class="xref" href="practical-scoring-function.html" title="Lucene 的实用评分函数">实用评分算法</a> 作为默认相似度算法，它也能够支持其他的一些算法，这些算法可以参考
<a href="/guide/en/elasticsearch/reference/5.6/index-modules-similarity.html#configuration" class="ulink" target="_top">相似度模块</a> 文档。</p>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="bm25"></a>Okapi BM25<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elasticsearch-cn/elasticsearch-definitive-guide/edit/cn/170_Relevance/70_Pluggable_similarities.asciidoc">edit</a>
</h3>
</div></div></div>
<p>能与 TF/IDF 和向量空间模型媲美的就是 <a href="http://en.wikipedia.org/wiki/Okapi_BM25" class="ulink" target="_top"><em>Okapi BM25</em></a> ，它被认为是 <em>当今最先进的</em> 排序函数。 BM25 源自 <a href="http://en.wikipedia.org/wiki/Probabilistic_relevance_model" class="ulink" target="_top">概率相关模型（probabilistic relevance model）</a> ，而不是向量空间模型，但这个算法也和 Lucene 的实用评分函数有很多共通之处。</p>
<p>BM25 同样使用词频、逆向文档频率以及字段长归一化，但是每个因子的定义都有细微区别。与其详细解释 BM25 公式，倒不如将关注点放在 BM25 所能带来的实际好处上。</p>
<div class="section">
<div class="titlepage"><div><div>
<h4 class="title">
<a id="bm25-saturation"></a>词频饱和度<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elasticsearch-cn/elasticsearch-definitive-guide/edit/cn/170_Relevance/70_Pluggable_similarities.asciidoc">edit</a>
</h4>
</div></div></div>
<p>TF/IDF 和 BM25 同样使用 <a class="xref" href="scoring-theory.html#idf" title="逆向文档频率">逆向文档频率</a> 来区分普通词（不重要）和非普通词（重要），同样认为（参见 <a class="xref" href="scoring-theory.html#tf" title="词频">词频</a> ）文档里的某个词出现次数越频繁，文档与这个词就越相关。</p>
<p>不幸的是，普通词随处可见，实际上一个普通词在同一个文档中大量出现的作用会由于该词在 <em>所有</em> 文档中的大量出现而被抵消掉。</p>
<p>曾经有个时期，将 <em>最</em> 普通的词（或 <em>停用词</em> ，参见 <a class="xref" href="stopwords.html" title="停用词: 性能与精度">停用词</a>）从索引中移除被认为是一种标准实践，TF/IDF 正是在这种背景下诞生的。TF/IDF 没有考虑词频上限的问题，因为高频停用词已经被移除了。</p>
<p>Elasticsearch 的 <code class="literal">standard</code> 标准分析器（ <code class="literal">string</code> 字段默认使用）不会移除停用词，因为尽管这些词的重要性很低，但也不是毫无用处。这导致：在一个相当长的文档中，像 <code class="literal">the</code> 和 <code class="literal">and</code> 这样词出现的数量会高得离谱，以致它们的权重被人为放大。</p>
<p>另一方面，BM25 有一个上限，文档里出现 5 到 10 次的词会比那些只出现一两次的对相关度有着显著影响。但是如图 <a class="xref" href="pluggable-similarites.html#img-bm25-saturation" title="TF/IDF 与 BM25 的词频饱和度">TF/IDF 与 BM25 的词频饱和度</a> 所见，文档中出现 20 次的词几乎与那些出现上千次的词有着相同的影响。</p>
<p>这就是 <em>非线性词频饱和度（nonlinear term-frequency saturation）</em> 。</p>
<div id="img-bm25-saturation" class="imageblock">
<div class="content">
<img src="images/elas_1706.png" alt="TF/IDF 与 BM25 的词频饱和度">
</div>
<div class="title">Figure 34. TF/IDF 与 BM25 的词频饱和度</div>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h4 class="title">
<a id="bm25-normalization"></a>字段长度归一化（Field-length normalization）<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elasticsearch-cn/elasticsearch-definitive-guide/edit/cn/170_Relevance/70_Pluggable_similarities.asciidoc">edit</a>
</h4>
</div></div></div>
<p>在 <a class="xref" href="scoring-theory.html#field-norm" title="字段长度归一值">字段长归一化</a> 中，我们提到过 Lucene 会认为较短字段比较长字段更重要：字段某个词的频度所带来的重要性会被这个字段长度抵消，但是实际的评分函数会将所有字段以同等方式对待。它认为所有较短的 <code class="literal">title</code> 字段比所有较长的 <code class="literal">body</code> 字段更重要。</p>
<p>BM25 当然也认为较短字段应该有更多的权重，但是它会分别考虑每个字段内容的平均长度，这样就能区分短 <code class="literal">title</code> 字段和 <code class="literal">长</code> title 字段。</p>
<div class="caution admon">
<div class="icon"></div>
<div class="admon_content">
<p>在 <a class="xref" href="query-time-boosting.html" title="查询时权重提升">查询时权重提升</a> 中，已经说过 <code class="literal">title</code> 字段因为其长度比 <code class="literal">body</code> 字段 <em>自然</em> 有更高的权重提升值。由于字段长度的差异只能应用于单字段，这种自然的权重提升会在使用 BM25 时消失。</p>
</div>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h4 class="title">
<a id="bm25-tunability"></a>BM25 调优<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elasticsearch-cn/elasticsearch-definitive-guide/edit/cn/170_Relevance/70_Pluggable_similarities.asciidoc">edit</a>
</h4>
</div></div></div>
<p>不像 TF/IDF ，BM25 有一个比较好的特性就是它提供了两个可调参数：</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<code class="literal">k1</code>
</span>
</dt>
<dd>
这个参数控制着词频结果在词频饱和度中的上升速度。默认值为 <code class="literal">1.2</code> 。值越小饱和度变化越快，值越大饱和度变化越慢。
</dd>
<dt>
<span class="term">
<code class="literal">b</code>
</span>
</dt>
<dd>
这个参数控制着字段长归一值所起的作用， <code class="literal">0.0</code> 会禁用归一化， <code class="literal">1.0</code> 会启用完全归一化。默认值为 <code class="literal">0.75</code> 。
</dd>
</dl>
</div>
<p>在实践中，调试 BM25 是另外一回事， <code class="literal">k1</code> 和 <code class="literal">b</code> 的默认值适用于绝大多数文档集合，但最优值还是会因为文档集不同而有所区别，为了找到文档集合的最优值，就必须对参数进行反复修改验证。</p>
</div>

</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="script-score.html">« 脚本评分</a>
</span>
<span class="next">
<a href="changing-similarities.html">更改相似度 »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
